Neuroevolution

  • Gonzalo Piérola
  • Iker García

Human Player

Run the following if you want to play the game and test It by yourself.
The objective is not to crash against the tubes.
The bird will jump if you push any key.

Credits: https://youtu.be/h2Uhla6nLDU


In [2]:
import pygame
from pygame.locals import *  # noqa
import sys
import random


class FlappyBird_Human:
    def __init__(self):
        self.screen = pygame.display.set_mode((400, 700))
        self.bird = pygame.Rect(65, 50, 50, 50)
        self.background = pygame.image.load("assets/background.png").convert()
        self.birdSprites = [pygame.image.load("assets/1.png").convert_alpha(),
                            pygame.image.load("assets/2.png").convert_alpha(),
                            pygame.image.load("assets/dead.png")]
        self.wallUp = pygame.image.load("assets/bottom.png").convert_alpha()
        self.wallDown = pygame.image.load("assets/top.png").convert_alpha()
        self.gap = 145
        self.wallx = 400
        self.birdY = 350
        self.jump = 0
        self.jumpSpeed = 15
        self.gravity = 10
        self.dead = False
        self.sprite = 0
        self.counter = 0
        self.offset = random.randint(-200, 200)

    def updateWalls(self):
        self.wallx -= 4
        if self.wallx < -80:
            self.wallx = 400
            self.counter += 1
            self.offset = random.randint(-200, 200)

    def birdUpdate(self):
        if self.jump:
            self.jumpSpeed -= 1
            self.birdY -= self.jumpSpeed
            self.jump -= 1
        else:
            self.birdY += self.gravity
            self.gravity += 0.2
        self.bird[1] = self.birdY
        upRect = pygame.Rect(self.wallx,
                             360 + self.gap - self.offset + 10,
                             self.wallUp.get_width() - 10,
                             self.wallUp.get_height())
        downRect = pygame.Rect(self.wallx,
                               0 - self.gap - self.offset - 10,
                               self.wallDown.get_width() - 10,
                               self.wallDown.get_height())
        if upRect.colliderect(self.bird):
            self.dead = True
        if downRect.colliderect(self.bird):
            self.dead = True
        if not 0 < self.bird[1] < 720:
            self.bird[1] = 50
            self.birdY = 50
            self.dead = False
            self.counter = 0
            self.wallx = 400
            self.offset = random.randint(-110, 110)
            self.gravity = 10

    def run(self):
        clock = pygame.time.Clock()
        pygame.font.init()
        font = pygame.font.SysFont("Arial", 50)
        while True:
            
            clock.tick(60)
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    sys.exit()
                if (event.type == pygame.KEYDOWN or event.type == pygame.MOUSEBUTTONDOWN) and not self.dead:
                    self.jump = 17
                    self.gravity = 10
                    self.jumpSpeed = 15

            self.screen.fill((255, 255, 255))
            self.screen.blit(self.background, (0, 0))
            self.screen.blit(self.wallUp,
                             (self.wallx, 360 + self.gap - self.offset))
            self.screen.blit(self.wallDown,
                             (self.wallx, 0 - self.gap - self.offset))
            self.screen.blit(font.render(str(self.counter),
                                         -1,
                                         (255, 255, 255)),
                             (200, 50))
            if self.dead:
                self.sprite = 2
            elif self.jump:
                self.sprite = 1
            self.screen.blit(self.birdSprites[self.sprite], (70, self.birdY))
            
            if not self.dead:
                self.sprite = 0
            self.updateWalls()
            self.birdUpdate()
            pygame.display.update()

if __name__ == "__main__":
    FlappyBird_Human().run()


pygame 1.9.4
Hello from the pygame community. https://www.pygame.org/contribute.html
---------------------------------------------------------------------------
error                                     Traceback (most recent call last)
<ipython-input-2-543e7677b2da> in <module>()
    101 
    102 if __name__ == "__main__":
--> 103     FlappyBird_Human().run()

<ipython-input-2-543e7677b2da> in __init__(self)
      7 class FlappyBird_Human:
      8     def __init__(self):
----> 9         self.screen = pygame.display.set_mode((400, 700))
     10         self.bird = pygame.Rect(65, 50, 50, 50)
     11         self.background = pygame.image.load("assets/background.png").convert()

error: No available video device

Game for training

The cell below contains a implementation of the game adapted to our needs.
The cell does not output any graphics
Every time the function "tick" is called, It executes a step of the game. This functions receives as a parameter if the bird will jump or no in that step.
We added functions needed to train our models


In [1]:
import pygame
from pygame.locals import *  # noqa
import sys
import random


class FlappyBird:
    
    def __init__(self):
       
        self.bird = pygame.Rect(65, 50, 50, 50)
        self.distance = 0
        self.gap = 145
        self.wallx = 400
        self.birdY = 350
        self.jump = 0
        self.jumpSpeed = 15
        self.gravity = 10
        self.dead = False
        self.counter = 0
        self.offset = random.randint(-200, 200)
        
    def calculateInput(self):
        dist_X_to_The_Wall = self.wallx+80
        dist_Y_to_The_Wall_UP = self.birdY-(0 - self.gap - self.offset+500)
        dist_Y_to_The_Wall_DOWN = self.birdY-(360 + self.gap - self.offset)
        dist_Y_TOP = self.birdY
        dist_Y_BOTTOM = 720-self.birdY
        res = [dist_X_to_The_Wall,dist_Y_to_The_Wall_UP,dist_Y_to_The_Wall_DOWN,dist_Y_TOP,dist_Y_BOTTOM]
        return res
    
    def centerWalls(self):
        return 0 - self.gap - self.offset+572.5
    
    def downWall(self):
        return 360 + self.gap - self.offset
    
    def posBird(self):
        return self.birdY
    
    def isDead(self):
        return self.dead
    
    def TotalDistance(self):
        return self.distance


    
    def updateWalls(self):
        self.wallx -= 4
        if self.wallx < -80:
            self.wallx = 400
            self.counter += 1
            self.offset = random.randint(-200, 200)

    def birdUpdate(self):
        self.distance =  self.distance + 1 
        if self.jump:
            self.jumpSpeed -= 1
            self.birdY -= self.jumpSpeed
            self.jump -= 1
        else:
            self.birdY += self.gravity
            self.gravity += 0.2
        self.bird[1] = self.birdY
        upRect = pygame.Rect(self.wallx,
                             360 + self.gap - self.offset + 10,
                             88,
                             500)
        downRect = pygame.Rect(self.wallx,
                               0 - self.gap - self.offset - 10,
                               88,
                               500)
        if upRect.colliderect(self.bird):
            self.dead = True
        if downRect.colliderect(self.bird):
            self.dead = True
        if not 0 < self.bird[1] < 720:
            self.dead=True
            
        

    def tick(self,jump):
        if (jump==True) and not self.dead:
            self.jump = 17
            self.gravity = 10
            self.jumpSpeed = 15
                
        self.updateWalls()
        self.birdUpdate()

Game with graphics

Similar to the previous cell, with this cell will show the bird playing the game.


In [2]:
import pygame
from pygame.locals import *  # noqa
import sys
import random

class FlappyBird_GAME:
    


    def __init__(self):
        self.screen = pygame.display.set_mode((400, 700))
        self.bird = pygame.Rect(65, 50, 50, 50)
        self.background = pygame.image.load("assets/background.png").convert()
        self.birdSprites = [pygame.image.load("assets/1.png").convert_alpha(),
                            pygame.image.load("assets/2.png").convert_alpha(),
                            pygame.image.load("assets/dead.png")]
        self.wallUp = pygame.image.load("assets/bottom.png").convert_alpha()
        self.wallDown = pygame.image.load("assets/top.png").convert_alpha()
        self.distance = 0
        self.gap = 145
        self.wallx = 400
        self.birdY = 350
        self.jump = 0
        self.jumpSpeed = 15
        self.gravity = 10
        self.dead = False
        self.counter = 0
        self.offset = random.randint(-200, 200)
        self.sprite = 0
        
    def calculateInput(self):
        dist_X_to_The_Wall = self.wallx+80
        dist_Y_to_The_Wall_UP = self.birdY-(0 - self.gap - self.offset+500)
        dist_Y_to_The_Wall_DOWN = self.birdY-(360 + self.gap - self.offset)
        dist_Y_TOP = self.birdY
        dist_Y_BOTTOM = 720-self.birdY
        res = [dist_X_to_The_Wall,dist_Y_to_The_Wall_UP,dist_Y_to_The_Wall_DOWN,dist_Y_TOP,dist_Y_BOTTOM]
        return res
    
    def isDead(self):
        return self.dead
    
    def TotalDistance(self):
        return self.distance

    def centerWalls(self):
        return 0 - self.gap - self.offset+572.5
    
    def downWall(self):
        return 360 + self.gap - self.offset
    
    def posBird(self):
        return self.birdY
    
    def updateWalls(self):
        self.wallx -= 4
        if self.wallx < -80:
            self.wallx = 400
            self.counter += 1
            self.offset = random.randint(-200, 200)

    def birdUpdate(self):
        self.distance =  self.distance + 1 
        if self.jump:
            self.jumpSpeed -= 1
            self.birdY -= self.jumpSpeed
            self.jump -= 1
        else:
            self.birdY += self.gravity
            self.gravity += 0.2
        self.bird[1] = self.birdY
        upRect = pygame.Rect(self.wallx,
                             360 + self.gap - self.offset + 10,
                             88,
                             500)
        downRect = pygame.Rect(self.wallx,
                               0 - self.gap - self.offset - 10,
                               88,
                               500)
        if upRect.colliderect(self.bird):
            self.dead = True
        if downRect.colliderect(self.bird):
            self.dead = True
        if not 0 < self.bird[1] < 720:
            self.dead=True
            
        

    def tick(self,jump):
        if (jump==True) and not self.dead:
                self.jump = 17
                self.gravity = 10
                self.jumpSpeed = 15
    

        self.screen.fill((255, 255, 255))
        self.screen.blit(self.background, (0, 0))
        self.screen.blit(self.wallUp,
                             (self.wallx, 360 + self.gap - self.offset))
    
        self.screen.blit(self.wallDown,
                             (self.wallx, 0 - self.gap - self.offset))
        self.screen.blit(font.render(str(self.counter),
                                         -1,
                                         (255, 255, 255)),
                             (200, 50))
        if self.dead:
            self.sprite = 2
        elif self.jump:
            self.sprite = 1
        self.screen.blit(self.birdSprites[self.sprite], (70, self.birdY))
        if not self.dead:
            self.sprite = 0
        self.updateWalls()
        self.birdUpdate()
        pygame.display.update()

Neuroevolution with NEAT

The cell below will evolve a network and Its weights to learn how to play the game The fitness will be the distance traveled by the bird.


In [3]:
import neat

number_generations = 1000
def eval_genomes(genomes,config):
    for genome_id, genome in genomes:
        genome.fitness = 99999
        net = neat.nn.FeedForwardNetwork.create(genome,config)
        bird = FlappyBird()
        while (not bird.isDead() and not bird.TotalDistance()>110000):
            nnInput = bird.calculateInput()
            #print(nnInput)
            #print(bird.fitness())
            output = net.activate(nnInput)
            if output[0] > output[1]:
                bird.tick(True)
            else:
                bird.tick(False)
            
        genome.fitness = bird.TotalDistance()
        
       
        

config = neat.Config(neat.DefaultGenome,neat.DefaultReproduction,neat.DefaultSpeciesSet,neat.DefaultStagnation,'FlapyBirdNEAT')

p = neat.Population(config)

p.add_reporter(neat.StdOutReporter(False))

winner = p.run(eval_genomes,number_generations)


 ****** Running generation 0 ****** 

Population's average fitness: 42.06000 stdev: 23.17965
Best fitness: 133.00000 - size: (2, 10) - species 1 - id 9
Average adjusted fitness: 0.158
Mean genetic distance 1.179, standard deviation 0.268
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.048 sec

 ****** Running generation 1 ****** 

Population's average fitness: 51.20000 stdev: 23.56438
Best fitness: 87.00000 - size: (3, 10) - species 1 - id 60
Average adjusted fitness: 0.423
Mean genetic distance 1.223, standard deviation 0.251
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.052 sec (0.050 average)

 ****** Running generation 2 ****** 

Population's average fitness: 57.38000 stdev: 32.34680
Best fitness: 202.00000 - size: (3, 10) - species 1 - id 143
Average adjusted fitness: 0.183
Mean genetic distance 1.362, standard deviation 0.244
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.063 sec (0.054 average)

 ****** Running generation 3 ****** 

Population's average fitness: 58.30000 stdev: 22.01022
Best fitness: 86.00000 - size: (2, 10) - species 1 - id 179
Average adjusted fitness: 0.546
Mean genetic distance 1.280, standard deviation 0.365
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.060 sec (0.056 average)

 ****** Running generation 4 ****** 

Population's average fitness: 69.50000 stdev: 40.82707
Best fitness: 223.00000 - size: (2, 8) - species 1 - id 212
Average adjusted fitness: 0.225
Mean genetic distance 1.438, standard deviation 0.348
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.069 sec (0.058 average)

 ****** Running generation 5 ****** 

Population's average fitness: 68.80000 stdev: 30.68680
Best fitness: 213.00000 - size: (2, 8) - species 1 - id 212
Average adjusted fitness: 0.233
Mean genetic distance 1.491, standard deviation 0.344
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.068 sec (0.060 average)

 ****** Running generation 6 ****** 

Population's average fitness: 63.42000 stdev: 30.71357
Best fitness: 203.00000 - size: (3, 9) - species 1 - id 305
Average adjusted fitness: 0.216
Mean genetic distance 1.573, standard deviation 0.425
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.062 sec (0.060 average)

 ****** Running generation 7 ****** 

Population's average fitness: 66.70000 stdev: 37.75725
Best fitness: 221.00000 - size: (2, 8) - species 1 - id 365
Average adjusted fitness: 0.213
Mean genetic distance 1.652, standard deviation 0.361
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.069 sec (0.061 average)

 ****** Running generation 8 ****** 

Population's average fitness: 74.82000 stdev: 61.88592
Best fitness: 451.00000 - size: (2, 8) - species 1 - id 370
Average adjusted fitness: 0.117
Mean genetic distance 1.487, standard deviation 0.445
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.079 sec (0.063 average)

 ****** Running generation 9 ****** 

Population's average fitness: 111.68000 stdev: 240.60062
Best fitness: 1773.00000 - size: (2, 8) - species 1 - id 482
Average adjusted fitness: 0.050
Mean genetic distance 1.587, standard deviation 0.466
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.097 sec (0.067 average)

 ****** Running generation 10 ****** 

Population's average fitness: 89.72000 stdev: 77.33461
Best fitness: 457.00000 - size: (2, 7) - species 1 - id 523
Average adjusted fitness: 0.150
Mean genetic distance 1.475, standard deviation 0.437
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.077 sec (0.069 average)

 ****** Running generation 11 ****** 

Population's average fitness: 83.20000 stdev: 64.44284
Best fitness: 444.00000 - size: (2, 6) - species 1 - id 507
Average adjusted fitness: 0.139
Mean genetic distance 1.483, standard deviation 0.404
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.081 sec (0.072 average)

 ****** Running generation 12 ****** 

Population's average fitness: 95.50000 stdev: 94.48645
Best fitness: 704.00000 - size: (2, 6) - species 1 - id 507
Average adjusted fitness: 0.104
Mean genetic distance 1.645, standard deviation 0.356
Population of 50 members in 1 species
Total extinctions: 0
Generation time: 0.090 sec (0.075 average)

 ****** Running generation 13 ****** 

Population's average fitness: 4586.06000 stdev: 21531.54082
Best fitness: 110001.00000 - size: (2, 5) - species 1 - id 639

Best individual in generation 13 meets fitness threshold - complexity: (2, 5)

Draw the network

We are using the graphviz in order to draw the network that we generate with the NEAT


In [4]:
import graphviz

def draw_net(config, genome, view=False, filename=None, node_names=None, show_disabled=True, prune_unused=False,
             node_colors=None, fmt='svg'):
    """ Receives a genome and draws a neural network with arbitrary topology. """
    # Attributes for network nodes.
    if graphviz is None:
        warnings.warn("This display is not available due to a missing optional dependency (graphviz)")
        return

    if node_names is None:
        node_names = {}

    assert type(node_names) is dict

    if node_colors is None:
        node_colors = {}

    assert type(node_colors) is dict

    node_attrs = {
        'shape': 'circle',
        'fontsize': '9',
        'height': '0.2',
        'width': '0.2'}

    dot = graphviz.Digraph(format=fmt, node_attr=node_attrs)

    inputs = set()
    for k in config.genome_config.input_keys:
        inputs.add(k)
        name = node_names.get(k, str(k))
        input_attrs = {'style': 'filled',
                       'shape': 'box'}
        input_attrs['fillcolor'] = node_colors.get(k, 'lightgray')
        dot.node(name, _attributes=input_attrs)

    outputs = set()
    for k in config.genome_config.output_keys:
        outputs.add(k)
        name = node_names.get(k, str(k))
        node_attrs = {'style': 'filled'}
        node_attrs['fillcolor'] = node_colors.get(k, 'lightblue')

        dot.node(name, _attributes=node_attrs)

    if prune_unused:
        connections = set()
        for cg in genome.connections.values():
            if cg.enabled or show_disabled:
                connections.add((cg.in_node_id, cg.out_node_id))

        used_nodes = copy.copy(outputs)
        pending = copy.copy(outputs)
        while pending:
            new_pending = set()
            for a, b in connections:
                if b in pending and a not in used_nodes:
                    new_pending.add(a)
                    used_nodes.add(a)
            pending = new_pending
    else:
        used_nodes = set(genome.nodes.keys())

    for n in used_nodes:
        if n in inputs or n in outputs:
            continue

        attrs = {'style': 'filled',
                 'fillcolor': node_colors.get(n, 'white')}
        dot.node(str(n), _attributes=attrs)

    for cg in genome.connections.values():
        if cg.enabled or show_disabled:
            # if cg.input not in used_nodes or cg.output not in used_nodes:
            # continue
            input, output = cg.key
            a = node_names.get(input, str(input))
            b = node_names.get(output, str(output))
            style = 'solid' if cg.enabled else 'dotted'
            color = 'green' if cg.weight > 0 else 'red'
            width = str(0.1 + abs(cg.weight / 5.0))
            dot.edge(a, b, _attributes={'style': style, 'color': color, 'penwidth': width})

    dot.render(filename, view=view)

    return dot

Visualization of the network


In [5]:
draw_net(config, winner, view=True)


Out[5]:
%3 -1 -1 -2 -2 0 0 -2->0 1 1 -2->1 -3 -3 -3->0 -3->1 -4 -4 -4->0 -4->1 -5 -5 -5->0

Visualization of the best genome

This cell will show the best genome playing the game.

By adding the commented part the inputs and outputs generated during the process will be saved in csv file.


In [ ]:
clock = pygame.time.Clock()
pygame.font.init()
font = pygame.font.SysFont("Arial", 50)
bird = FlappyBird_GAME()

#import csv
#import numpy


while (not bird.isDead()):
    clock.tick(60)
    net = neat.nn.FeedForwardNetwork.create(winner,config)
    nnInput = bird.calculateInput()
    output = net.activate(nnInput)
    if output[0] > output[1]:
        bird.tick(True)
 #       out = [1.0,0.0]
    else:
        bird.tick(False)
  #      out = [0.0,1.0]
    
    #w = nnInput + out
    #a = numpy.asarray(w)
   # with open(r'flapyData.csv', 'a') as f:
     #   writer = csv.writer(f)
     #   writer.writerow(a)
print(bird.TotalDistance())

DEEP LEARNING

This cell will train a MLP with the data generated playing with the network generated by the NEAT to play the game.


In [6]:
import pandas as pd
import numpy as np
from sklearn import linear_model, datasets, metrics
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier

dat=pd.read_csv('flapyData.csv', sep=',',header=None)

inputs,outputs = np.column_stack((dat[0],dat[1],dat[2],dat[3],dat[4])),np.column_stack((dat[5],dat[6]))
X_train, X_test, Y_train, Y_test = train_test_split(inputs, outputs, test_size = 0.2, random_state=0)

mlp_classifier = MLPClassifier(hidden_layer_sizes=(10,4), max_iter=1000, tol=0.001, random_state=1, verbose=True)

mlp_classifier.fit(X_train,Y_train)


Iteration 1, loss = 2.34494169
Iteration 2, loss = 0.86819454
Iteration 3, loss = 0.34389195
Iteration 4, loss = 0.18168886
Iteration 5, loss = 0.12208448
Iteration 6, loss = 0.09701422
Iteration 7, loss = 0.08382616
Iteration 8, loss = 0.07319258
Iteration 9, loss = 0.06424246
Iteration 10, loss = 0.05851095
Iteration 11, loss = 0.05232148
Iteration 12, loss = 0.04579252
Iteration 13, loss = 0.04109867
Iteration 14, loss = 0.03649717
Iteration 15, loss = 0.03393865
Iteration 16, loss = 0.03240766
Iteration 17, loss = 0.02961006
Iteration 18, loss = 0.02762958
Iteration 19, loss = 0.02531280
Iteration 20, loss = 0.02375726
Iteration 21, loss = 0.02282959
Iteration 22, loss = 0.02146163
Iteration 23, loss = 0.02188076
Iteration 24, loss = 0.01948029
Iteration 25, loss = 0.01783538
Iteration 26, loss = 0.01670191
Iteration 27, loss = 0.01860552
Iteration 28, loss = 0.01608775
Iteration 29, loss = 0.01560375
Training loss did not improve more than tol=0.001000 for two consecutive epochs. Stopping.
Out[6]:
MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,
       beta_2=0.999, early_stopping=False, epsilon=1e-08,
       hidden_layer_sizes=(10, 4), learning_rate='constant',
       learning_rate_init=0.001, max_iter=1000, momentum=0.9,
       nesterovs_momentum=True, power_t=0.5, random_state=1, shuffle=True,
       solver='adam', tol=0.001, validation_fraction=0.1, verbose=True,
       warm_start=False)

Results

The results obtained classifying the test data with the trained MLP.


In [7]:
print("MLP predictions:\n%s\n" % (metrics.classification_report(Y_test, mlp_classifier.predict(X_test))))


MLP predictions:
             precision    recall  f1-score   support

          0       1.00      0.97      0.98       758
          1       1.00      1.00      1.00     11413

avg / total       1.00      1.00      1.00     12171


MLP playing the game

This cell will show how the MLP plays the game


In [ ]:
#Esta celda visualiza el mejor genoma tras la neuroevolución

clock = pygame.time.Clock()
pygame.font.init()
font = pygame.font.SysFont("Arial", 50)
bird = FlappyBird_GAME()


while (not bird.isDead()):
    clock.tick(60)
    nnInput = bird.calculateInput()
    output =  mlp_classifier.predict(np.column_stack((nnInput[0],nnInput[1],nnInput[2],nnInput[3],nnInput[4])))
    if output[0][0] > output[0][1]:
        bird.tick(True)
    else:
        bird.tick(False)

print(bird.TotalDistance())